{
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "# And for visualization on Colab install\n",
        "!apt-get install x11-utils > /dev/null 2>&1 \n",
        "!pip install pyglet\n",
        "!apt-get install -y xvfb python-opengl > /dev/null 2>&1\n",
        "!pip install gym pyvirtualdisplay > /dev/null 2>&1"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fQ3et1ApnNYT",
        "outputId": "9d57eb70-aeae-495e-fd07-2a5d36de608a"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: pyglet in /usr/local/lib/python3.7/dist-packages (1.5.0)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from pyglet) (0.16.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!wget http://www.atarimania.com/roms/Roms.rar\n",
        "!mkdir /content/ROM/\n",
        "!unrar e /content/Roms.rar /content/ROM/\n",
        "!python -m atari_py.import_roms /content/ROM/"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4Crum-mFsLvz",
        "outputId": "ae3476e7-17b8-4a36-9cff-799280e4d593"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2022-03-26 13:28:51--  http://www.atarimania.com/roms/Roms.rar\n",
            "Resolving www.atarimania.com (www.atarimania.com)... 195.154.81.199\n",
            "Connecting to www.atarimania.com (www.atarimania.com)|195.154.81.199|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 11128004 (11M) [application/x-rar-compressed]\n",
            "Saving to: ‘Roms.rar.1’\n",
            "\n",
            "Roms.rar.1          100%[===================>]  10.61M   485KB/s    in 23s     \n",
            "\n",
            "2022-03-26 13:29:15 (468 KB/s) - ‘Roms.rar.1’ saved [11128004/11128004]\n",
            "\n",
            "mkdir: cannot create directory ‘/content/ROM/’: File exists\n",
            "\n",
            "UNRAR 5.50 freeware      Copyright (c) 1993-2017 Alexander Roshal\n",
            "\n",
            "\n",
            "Extracting from /content/Roms.rar\n",
            "\n",
            "\n",
            "Would you like to replace the existing file /content/ROM/HC ROMS.zip\n",
            "11826711 bytes, modified on 2019-12-22 11:24\n",
            "with a new one\n",
            "11826711 bytes, modified on 2019-12-22 11:24\n",
            "\n",
            "[Y]es, [N]o, [A]ll, n[E]ver, [R]ename, [Q]uit Q\n",
            "\n",
            "Program aborted\n",
            "copying adventure.bin from HC ROMS/BY ALPHABET (PAL)/A-G/Adventure (PAL).bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/adventure.bin\n",
            "copying air_raid.bin from HC ROMS/BY ALPHABET (PAL)/A-G/Air Raid (PAL).bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/air_raid.bin\n",
            "copying alien.bin from HC ROMS/BY ALPHABET (PAL)/A-G/REMAINING NTSC ORIGINALS/Alien.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/alien.bin\n",
            "copying crazy_climber.bin from HC ROMS/BY ALPHABET (PAL)/A-G/REMAINING NTSC ORIGINALS/Crazy Climber.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/crazy_climber.bin\n",
            "copying elevator_action.bin from HC ROMS/BY ALPHABET (PAL)/A-G/REMAINING NTSC ORIGINALS/Elevator Action (Prototype).bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/elevator_action.bin\n",
            "copying gravitar.bin from HC ROMS/BY ALPHABET (PAL)/A-G/REMAINING NTSC ORIGINALS/Gravitar.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/gravitar.bin\n",
            "copying keystone_kapers.bin from HC ROMS/BY ALPHABET (PAL)/H-R/Keystone Kapers (PAL).bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/keystone_kapers.bin\n",
            "copying king_kong.bin from HC ROMS/BY ALPHABET (PAL)/H-R/King Kong (PAL).bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/king_kong.bin\n",
            "copying laser_gates.bin from HC ROMS/BY ALPHABET (PAL)/H-R/Laser Gates (PAL).bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/laser_gates.bin\n",
            "copying mr_do.bin from HC ROMS/BY ALPHABET (PAL)/H-R/Mr. Do! (PAL).bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/mr_do.bin\n",
            "copying pacman.bin from HC ROMS/BY ALPHABET (PAL)/H-R/Pac-Man (PAL).bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/pacman.bin\n",
            "copying jamesbond.bin from HC ROMS/BY ALPHABET (PAL)/H-R/REMAINING NTSC ORIGINALS/James Bond 007.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/jamesbond.bin\n",
            "copying koolaid.bin from HC ROMS/BY ALPHABET (PAL)/H-R/REMAINING NTSC ORIGINALS/Kool-Aid Man.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/koolaid.bin\n",
            "copying krull.bin from HC ROMS/BY ALPHABET (PAL)/H-R/REMAINING NTSC ORIGINALS/Krull.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/krull.bin\n",
            "copying montezuma_revenge.bin from HC ROMS/BY ALPHABET (PAL)/H-R/REMAINING NTSC ORIGINALS/Montezuma's Revenge - Featuring Panama Joe.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/montezuma_revenge.bin\n",
            "copying star_gunner.bin from HC ROMS/BY ALPHABET (PAL)/S-Z/REMAINING NTSC ORIGINALS/Stargunner.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/star_gunner.bin\n",
            "copying time_pilot.bin from HC ROMS/BY ALPHABET (PAL)/S-Z/REMAINING NTSC ORIGINALS/Time Pilot.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/time_pilot.bin\n",
            "copying up_n_down.bin from HC ROMS/BY ALPHABET (PAL)/S-Z/REMAINING NTSC ORIGINALS/Up 'n Down.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/up_n_down.bin\n",
            "copying sir_lancelot.bin from HC ROMS/BY ALPHABET (PAL)/S-Z/Sir Lancelot (PAL).bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/sir_lancelot.bin\n",
            "copying amidar.bin from HC ROMS/BY ALPHABET/A-G/Amidar.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/amidar.bin\n",
            "copying asteroids.bin from HC ROMS/BY ALPHABET/A-G/Asteroids [no copyright].bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/asteroids.bin\n",
            "copying atlantis.bin from HC ROMS/BY ALPHABET/A-G/Atlantis.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/atlantis.bin\n",
            "copying bank_heist.bin from HC ROMS/BY ALPHABET/A-G/Bank Heist.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/bank_heist.bin\n",
            "copying battle_zone.bin from HC ROMS/BY ALPHABET/A-G/Battlezone.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/battle_zone.bin\n",
            "copying beam_rider.bin from HC ROMS/BY ALPHABET/A-G/Beamrider.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/beam_rider.bin\n",
            "copying berzerk.bin from HC ROMS/BY ALPHABET/A-G/Berzerk.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/berzerk.bin\n",
            "copying bowling.bin from HC ROMS/BY ALPHABET/A-G/Bowling.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/bowling.bin\n",
            "copying boxing.bin from HC ROMS/BY ALPHABET/A-G/Boxing.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/boxing.bin\n",
            "copying breakout.bin from HC ROMS/BY ALPHABET/A-G/Breakout - Breakaway IV.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/breakout.bin\n",
            "copying carnival.bin from HC ROMS/BY ALPHABET/A-G/Carnival.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/carnival.bin\n",
            "copying centipede.bin from HC ROMS/BY ALPHABET/A-G/Centipede.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/centipede.bin\n",
            "copying chopper_command.bin from HC ROMS/BY ALPHABET/A-G/Chopper Command.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/chopper_command.bin\n",
            "copying defender.bin from HC ROMS/BY ALPHABET/A-G/Defender.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/defender.bin\n",
            "copying demon_attack.bin from HC ROMS/BY ALPHABET/A-G/Demon Attack.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/demon_attack.bin\n",
            "copying donkey_kong.bin from HC ROMS/BY ALPHABET/A-G/Donkey Kong.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/donkey_kong.bin\n",
            "copying double_dunk.bin from HC ROMS/BY ALPHABET/A-G/Double Dunk.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/double_dunk.bin\n",
            "copying enduro.bin from HC ROMS/BY ALPHABET/A-G/Enduro.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/enduro.bin\n",
            "copying fishing_derby.bin from HC ROMS/BY ALPHABET/A-G/Fishing Derby.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/fishing_derby.bin\n",
            "copying freeway.bin from HC ROMS/BY ALPHABET/A-G/Freeway.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/freeway.bin\n",
            "copying frogger.bin from HC ROMS/BY ALPHABET/A-G/Frogger.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/frogger.bin\n",
            "copying frostbite.bin from HC ROMS/BY ALPHABET/A-G/Frostbite.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/frostbite.bin\n",
            "copying galaxian.bin from HC ROMS/BY ALPHABET/A-G/Galaxian.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/galaxian.bin\n",
            "copying gopher.bin from HC ROMS/BY ALPHABET/A-G/Gopher.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/gopher.bin\n",
            "copying hero.bin from HC ROMS/BY ALPHABET/H-R/H.E.R.O..bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/hero.bin\n",
            "copying ice_hockey.bin from HC ROMS/BY ALPHABET/H-R/Ice Hockey.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/ice_hockey.bin\n",
            "copying journey_escape.bin from HC ROMS/BY ALPHABET/H-R/Journey Escape.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/journey_escape.bin\n",
            "copying kaboom.bin from HC ROMS/BY ALPHABET/H-R/Kaboom!.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/kaboom.bin\n",
            "copying kangaroo.bin from HC ROMS/BY ALPHABET/H-R/Kangaroo.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/kangaroo.bin\n",
            "copying kung_fu_master.bin from HC ROMS/BY ALPHABET/H-R/Kung-Fu Master.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/kung_fu_master.bin\n",
            "copying lost_luggage.bin from HC ROMS/BY ALPHABET/H-R/Lost Luggage [no opening scene].bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/lost_luggage.bin\n",
            "copying ms_pacman.bin from HC ROMS/BY ALPHABET/H-R/Ms. Pac-Man.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/ms_pacman.bin\n",
            "copying name_this_game.bin from HC ROMS/BY ALPHABET/H-R/Name This Game.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/name_this_game.bin\n",
            "copying phoenix.bin from HC ROMS/BY ALPHABET/H-R/Phoenix.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/phoenix.bin\n",
            "copying pitfall.bin from HC ROMS/BY ALPHABET/H-R/Pitfall! - Pitfall Harry's Jungle Adventure.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/pitfall.bin\n",
            "copying pooyan.bin from HC ROMS/BY ALPHABET/H-R/Pooyan.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/pooyan.bin\n",
            "copying private_eye.bin from HC ROMS/BY ALPHABET/H-R/Private Eye.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/private_eye.bin\n",
            "copying qbert.bin from HC ROMS/BY ALPHABET/H-R/Q-bert.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/qbert.bin\n",
            "copying riverraid.bin from HC ROMS/BY ALPHABET/H-R/River Raid.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/riverraid.bin\n",
            "copying road_runner.bin from patched version of HC ROMS/BY ALPHABET/H-R/Road Runner.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/road_runner.bin\n",
            "copying robotank.bin from HC ROMS/BY ALPHABET/H-R/Robot Tank.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/robotank.bin\n",
            "copying seaquest.bin from HC ROMS/BY ALPHABET/S-Z/Seaquest.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/seaquest.bin\n",
            "copying skiing.bin from HC ROMS/BY ALPHABET/S-Z/Skiing.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/skiing.bin\n",
            "copying solaris.bin from HC ROMS/BY ALPHABET/S-Z/Solaris.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/solaris.bin\n",
            "copying space_invaders.bin from HC ROMS/BY ALPHABET/S-Z/Space Invaders.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/space_invaders.bin\n",
            "copying surround.bin from HC ROMS/BY ALPHABET/S-Z/Surround - Chase.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/surround.bin\n",
            "copying tennis.bin from HC ROMS/BY ALPHABET/S-Z/Tennis.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/tennis.bin\n",
            "copying trondead.bin from HC ROMS/BY ALPHABET/S-Z/TRON - Deadly Discs.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/trondead.bin\n",
            "copying tutankham.bin from HC ROMS/BY ALPHABET/S-Z/Tutankham.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/tutankham.bin\n",
            "copying venture.bin from HC ROMS/BY ALPHABET/S-Z/Venture.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/venture.bin\n",
            "copying pong.bin from HC ROMS/BY ALPHABET/S-Z/Video Olympics - Pong Sports.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/pong.bin\n",
            "copying video_pinball.bin from HC ROMS/BY ALPHABET/S-Z/Video Pinball - Arcade Pinball.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/video_pinball.bin\n",
            "copying wizard_of_wor.bin from HC ROMS/BY ALPHABET/S-Z/Wizard of Wor.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/wizard_of_wor.bin\n",
            "copying yars_revenge.bin from HC ROMS/BY ALPHABET/S-Z/Yars' Revenge.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/yars_revenge.bin\n",
            "copying zaxxon.bin from HC ROMS/BY ALPHABET/S-Z/Zaxxon.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/zaxxon.bin\n",
            "copying assault.bin from HC ROMS/NTSC VERSIONS OF PAL ORIGINALS/Assault (AKA Sky Alien) (1983) (Bomb - Onbase) (CA281).bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/assault.bin\n",
            "copying asterix.bin from ROMS/Asterix (AKA Taz) (07-27-1983) (Atari, Jerome Domurat, Steve Woita) (CX2696) (Prototype).bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/asterix.bin\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "R1utWUXDnJ9I"
      },
      "outputs": [],
      "source": [
        "import random\n",
        "import gym\n",
        "#import math\n",
        "import numpy as np\n",
        "from collections import deque\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense, Conv2D, MaxPooling2D, Flatten\n",
        "from tensorflow.keras.optimizers import Adam"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "## Uncomment if working on Colab\n",
        "from pyvirtualdisplay import Display\n",
        "display = Display(visible=0, size=(600, 400))\n",
        "display.start()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3oWLRo7snWUK",
        "outputId": "fe5d76d5-d93b-417a-eca9-0c68518a05d7"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<pyvirtualdisplay.display.Display at 0x7f6dc518cc10>"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "LyUsgZfDnJ9K"
      },
      "outputs": [],
      "source": [
        "EPOCHS = 500\n",
        "THRESHOLD = 10\n",
        "MONITOR = True"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "FH8YZhOpnJ9L"
      },
      "outputs": [],
      "source": [
        "class DQN():\n",
        "    def __init__(self, env_string,batch_size=64, IM_SIZE = 84, m = 4):\n",
        "        self.memory = deque(maxlen=5000)\n",
        "        self.env = gym.make(env_string)\n",
        "        input_size = self.env.observation_space.shape[0]\n",
        "        action_size = self.env.action_space.n\n",
        "        self.batch_size = batch_size\n",
        "        self.gamma = 1.0\n",
        "        self.epsilon = 1.0\n",
        "        self.epsilon_min = 0.01\n",
        "        self.epsilon_decay = 0.995\n",
        "        self.IM_SIZE = IM_SIZE\n",
        "        self.m = m\n",
        "       \n",
        "        \n",
        "        alpha=0.01\n",
        "        alpha_decay=0.01\n",
        "        if MONITOR: self.env = gym.wrappers.Monitor(self.env, '../data/'+env_string, force=True)\n",
        "        \n",
        "        # Init model\n",
        "        self.model = Sequential()\n",
        "        self.model.add( Conv2D(32, 8, (4,4), activation='relu',padding='valid', input_shape=(IM_SIZE, IM_SIZE, m)))\n",
        "        #self.model.add(MaxPooling2D())\n",
        "        self.model.add( Conv2D(64, 4, (2,2), activation='relu',padding='valid'))\n",
        "        self.model.add(MaxPooling2D())\n",
        "        self.model.add( Conv2D(64, 3, (1,1), activation='relu',padding='valid'))\n",
        "        self.model.add(MaxPooling2D())\n",
        "        self.model.add(Flatten())\n",
        "        self.model.add(Dense(256, activation='elu'))\n",
        "        self.model.add(Dense(action_size, activation='linear'))\n",
        "        self.model.compile(loss='mse', optimizer=Adam(lr=alpha, decay=alpha_decay))\n",
        "        self.model_target = tf.keras.models.clone_model(self.model)\n",
        "\n",
        "    def remember(self, state, action, reward, next_state, done):\n",
        "        self.memory.append((state, action, reward, next_state, done))\n",
        "\n",
        "    def choose_action(self, state, epsilon):\n",
        "        if np.random.random() <= epsilon:\n",
        "            return self.env.action_space.sample()\n",
        "        else:\n",
        "            return np.argmax(self.model.predict(state))\n",
        "\n",
        "    def preprocess_state(self, img):\n",
        "        img_temp = img[31:195]  # Choose the important area of the image\n",
        "        img_temp = tf.image.rgb_to_grayscale(img_temp)\n",
        "        img_temp = tf.image.resize(img_temp, [self.IM_SIZE, self.IM_SIZE],\n",
        "                                   method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)\n",
        "        img_temp = tf.cast(img_temp, tf.float32)\n",
        "        return img_temp[:,:,0]\n",
        "\n",
        "    def combine_images(self, img1, img2):\n",
        "        if len(img1.shape) == 3 and img1.shape[0] == self.m:\n",
        "            im = np.append(img1[1:,:, :],np.expand_dims(img2,0), axis=2)\n",
        "            return tf.expand_dims(im, 0)\n",
        "        else:\n",
        "            im = np.stack([img1]*self.m, axis = 2)\n",
        "            return tf.expand_dims(im, 0)\n",
        "        #return np.reshape(state, [1, 4])\n",
        "\n",
        "    def replay(self, batch_size):\n",
        "        x_batch, y_batch = [], []\n",
        "        minibatch = random.sample(self.memory, min(len(self.memory), batch_size))\n",
        "        for state, action, reward, next_state, done in minibatch:\n",
        "            y_target = self.model_target.predict(state)\n",
        "            y_target[0][action] = reward if done else reward + self.gamma * np.max(self.model.predict(next_state)[0])\n",
        "            x_batch.append(state[0])\n",
        "            y_batch.append(y_target[0])\n",
        "        \n",
        "        self.model.fit(np.array(x_batch), np.array(y_batch), batch_size=len(x_batch), verbose=0)\n",
        "        #epsilon = max(epsilon_min, epsilon_decay*epsilon) # decrease epsilon\n",
        "       \n",
        "\n",
        "    def train(self):\n",
        "        scores = deque(maxlen=100)\n",
        "        avg_scores = []\n",
        "        \n",
        "\n",
        "        for e in range(EPOCHS):\n",
        "            state = self.env.reset()\n",
        "            state = self.preprocess_state(state)\n",
        "            state = self.combine_images(state, state)\n",
        "            done = False\n",
        "            i = 0\n",
        "            while not done:\n",
        "                action = self.choose_action(state,self.epsilon)\n",
        "                next_state, reward, done, _ = self.env.step(action)\n",
        "                next_state = self.preprocess_state(next_state)\n",
        "                next_state = self.combine_images(next_state, state)\n",
        "                #print(next_state.shape)\n",
        "                self.remember(state, action, reward, next_state, done)\n",
        "                state = next_state\n",
        "                self.epsilon = max(self.epsilon_min, self.epsilon_decay*self.epsilon) # decrease epsilon\n",
        "                i += reward\n",
        "\n",
        "            scores.append(i)\n",
        "            mean_score = np.mean(scores)\n",
        "            avg_scores.append(mean_score)\n",
        "            if mean_score >= THRESHOLD:\n",
        "                print('Solved after {} trials ✔'.format(e))\n",
        "                return avg_scores\n",
        "            if e % 10 == 0:\n",
        "                print('[Episode {}] - Average Score: {}.'.format(e, mean_score))\n",
        "                self.model_target.set_weights(self.model.get_weights())\n",
        "\n",
        "            self.replay(self.batch_size)\n",
        "        \n",
        "        print('Did not solve after {} episodes 😞'.format(e))\n",
        "        return avg_scores\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lhA_8CqPnJ9N",
        "outputId": "75063ecd-179c-4d0a-be77-5bf825aaa4f3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        }
      ],
      "source": [
        "env_string = 'BreakoutDeterministic-v4'\n",
        "agent = DQN(env_string)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L1P006i5nJ9N",
        "outputId": "a7b50bd2-c3c6-4551-aa83-42db05076a1c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv2d (Conv2D)             (None, 20, 20, 32)        8224      \n",
            "                                                                 \n",
            " conv2d_1 (Conv2D)           (None, 9, 9, 64)          32832     \n",
            "                                                                 \n",
            " max_pooling2d (MaxPooling2D  (None, 4, 4, 64)         0         \n",
            " )                                                               \n",
            "                                                                 \n",
            " conv2d_2 (Conv2D)           (None, 2, 2, 64)          36928     \n",
            "                                                                 \n",
            " max_pooling2d_1 (MaxPooling  (None, 1, 1, 64)         0         \n",
            " 2D)                                                             \n",
            "                                                                 \n",
            " flatten (Flatten)           (None, 64)                0         \n",
            "                                                                 \n",
            " dense (Dense)               (None, 256)               16640     \n",
            "                                                                 \n",
            " dense_1 (Dense)             (None, 4)                 1028      \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 95,652\n",
            "Trainable params: 95,652\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "agent.model.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eUdvGyeKnJ9O",
        "outputId": "bdf98041-3a4f-45b5-e3d1-b7a74a34eeee"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv2d (Conv2D)             (None, 20, 20, 32)        8224      \n",
            "                                                                 \n",
            " conv2d_1 (Conv2D)           (None, 9, 9, 64)          32832     \n",
            "                                                                 \n",
            " max_pooling2d (MaxPooling2D  (None, 4, 4, 64)         0         \n",
            " )                                                               \n",
            "                                                                 \n",
            " conv2d_2 (Conv2D)           (None, 2, 2, 64)          36928     \n",
            "                                                                 \n",
            " max_pooling2d_1 (MaxPooling  (None, 1, 1, 64)         0         \n",
            " 2D)                                                             \n",
            "                                                                 \n",
            " flatten (Flatten)           (None, 64)                0         \n",
            "                                                                 \n",
            " dense (Dense)               (None, 256)               16640     \n",
            "                                                                 \n",
            " dense_1 (Dense)             (None, 4)                 1028      \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 95,652\n",
            "Trainable params: 95,652\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "agent.model_target.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wjuavdKxnJ9P",
        "outputId": "f326272a-0c5a-410d-e84c-a77086c3f5ab"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[Episode 0] - Average Score: 3.0.\n",
            "[Episode 10] - Average Score: 8.272727272727273.\n",
            "[Episode 20] - Average Score: 9.571428571428571.\n",
            "[Episode 30] - Average Score: 9.935483870967742.\n",
            "Solved after 32 trials ✔\n"
          ]
        }
      ],
      "source": [
        "scores = agent.train()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "mUUBXJrHnJ9P",
        "outputId": "eaf87ae9-af2d-483b-c90f-fbe97203043a"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxcZ33v8c+jfRntGkm2ZFm25S22YzsWibPaJGULZKFAbihLCjSmSWnpbXm10Mu9SS/Q20LbSykQapaSkDQUyNoUQtysEC+J7CjxbkvyKlnbaJ3RMtLM0z9m5Mi2JEuasUZn5vt+vfzS6OjMnF/OK/r68XOexVhrERER50mKdQEiIjIzCnAREYdSgIuIOJQCXETEoRTgIiIOlTKbFysuLrZVVVWzeUkREcfbvXt3h7XWff7xWQ3wqqoqamtrZ/OSIiKOZ4w5Md5xdaGIiDiUAlxExKEU4CIiDqUAFxFxKAW4iIhDXTTAjTE/Msa0GWP2jTlWaIzZZow5Gv5acGnLFBGR802lBf5j4L3nHfsi8Ly1dinwfPh7ERGZRRcdB26tfcUYU3Xe4duAzeHXDwIvAX8ZxbpERBzHWkvf0AjtfUN09A3R7h2ivS/057M3LCEvKzWq15vpRJ5Sa+2Z8OsWoHSiE40xW4AtAJWVlTO8nIhI7ASCFo93iLa+IVp7B2nrG6Ktd4jWvsFQWI8J6qGR4AXvT04y3L6+fM4E+FnWWmuMmXBXCGvtVmArQE1NjXaPEElw1lq6+odp6hqgqbuffn+Aa6uLKc3NiEkt3f3DtPQO0tI7SFvvIC09Q2dft/YN0tYbCujgOOlVmJ1GSU467px0qoqyceek43alU5yThtuVEfo+J538zFSSkkzU659pgLcaY+ZZa88YY+YBbdEsSkScLRC0HGrppbHdR1P3AKe7+mnqGuB01wBN3QP0+wMXvGfdgnzedVkp71lVyhK3C2MiCzxrLR6fnzPdgzT3DHCme4AzPYOc6RmkpScU2K29g+O2mIuy0yjNzaA0N51V8/IoyU2nJDeDkpx0SnLSKc3NoNiVTlpKbAfyzTTAnwbuAv42/PWpqFUkIo4TDFoOtvSys7GTHQ0eXjvmoXdw5OzP8zJTqSjIZFFxNtcvdVNekElFQSbl+ZkYAy8eamPbgVa+8evDfOPXh1lUnM27LyvlXZeVsr6ygORxWq+DwwGaugfCLfnQ1+bugVBYh4Paf144pyUnUZaXQVleBusW5FOWl3E2qMtyQ69LctNJT0m+5PcsGszF9sQ0xjxK6IFlMdAK3Ac8CfwMqAROAHdYazsvdrGamhqrxaxEnC8YtBxu7WNHg4edjR52HeukZ2AYgIVFWWxcVMTGJYWsnJdLeX4mORlT6/tt6Rlk28FWntvfws5GD8MBS7ErjZtWlJKTkRIK6nBYe3z+c96bnGQoy81gXl4G8/IzmZ839nUmZXkZFGWnXZKujEvNGLPbWltzwfHZ3NRYAS7iLP6RIKe6+jnW7uNYh4/GDh/HOrwcaumjuz8U2JWFWVy9OBTYVy0qYn5+ZlSu3Ts4zEuH29l2oJWXDrUxHAwyPz/Uah9tvZcXZFKen0V5QSalOemkJMfn3MSJAnxWl5MVkblntK/4aKuX+nZvOKy9HOvwcaprgMCYp3eF2WksKs7mPZeVcdXiQq5aXER5lAL7fLkZqdy6dj63rp1PMGgxhoj7xeONAlwkQQSDluaeAerbvOf+afeebU0DZKUls6g4m9Xledy6dj5VxdksCv/Jz0qLSe1O7PaYDQpwkTjU4R3icEsfh1r6ONzSy+GWPo62ec8Z/VGYnUZ1iYub18yj2u1iaamL6hIXZbkZauk6hAJcxMEGhwMcae3j0JlwWLeGwrrD+/YDvqLsNJaX5XBHzYJQSLtDQV3kSo9h5RINCnARh/AOjXCguZf9zT3sawp9rW/zMhLuo05PSWJZaQ6bl5ewoiyHFWW5LC/LwZ2joI5XCnCROah3cJi9p3t463QP+5p7ONDcy7EO39mfF7vSWDU/jxtXlLBqfh4r5+WwsCh73PHSEr8U4CIx5h8Jcrilj7pTXdSd6uHN0900tHsZHeFbUZDJqvm5fHB9OavLc1k1P4+SnHT1U4sCXGS2NXcP8PrxTupOdfPmqW72NfeenTFYlJ3GugX53Lp2PusW5HN5RV7MRn7I3KcAF7mEgkHLkbY+Xj/eRe3xTmqPd9HUPQBAZmoya8rzuOvqhaxbUMDaBXnhqeVqWcvUKMBFomhwOMCbp7qpPdHF68c72X2ii77wmiAlOem8Y1Ehd1+/iJqqQlaU5cTtzEGZHQpwkQgEgpa9TT28Wt/B9oYOao93nV3dbmmJiw9cPp93VBXwjqpCKgrUupboUoCLTIO1lqNtXl6t7+DVeg+7jnnOtrCXl+bwe1dVcvXiIt5RVUhBtvqu5dJSgItcRJfPz8tH2nnxcBuv1nvo8A4BoUWc3r9mHtdUF3P14iKNt5ZZpwAXOY+1lv3Nvbx0uI0XDrVRd6qboA2NELluaTHXLCnimiXFLCjMinWpkuAU4CKEZjn+9mgHLx5q48XDbbT1hVrZl1fk8cc3LuWdK0q4vDxPiyrJnKIAl4TV6fOz7UALv9zbwvaGDoYDlpz0FG5Y5mbzcjebl5eoW0TmNAW4JJT2viF+vb+FX+07w87GTgJBS2VhFp+6dhE3rihhw8ICUjW0TxxCAS5xr7V3kGf3tfDLvWd47Xgn1sLi4mzu2bSE960p47J5uRreJ46kAJe41DMwzH++dYYn3jjN68e7AFhW6uJPblzKzWvmsaw08l3PRWItogA3xnweuBswwPettd+MSlUiMzASCPKbox38Ys9pth1oxT8SZGmJiy+8exnvXT2P6hJXrEsUiaoZB7gxZjWh8L4S8APPGmOesdbWR6s4kak41NLLY7tP82RdM+19QxRkpfLRdyzgQxsqWFOep5a2xK1IWuArgV3W2n4AY8zLwO8CX49GYSKT6fL5eeKNJh7bc5r9zb2kJBneuaKED11RwY0rSkhL0YNIiX+RBPg+4GvGmCJgALgZqD3/JGPMFmALQGVlZQSXE4H9zT08uP04T9U1MzQSZHV5Lvfdchm3rp2vLcIk4cw4wK21B40xfwc8B/iAOiAwznlbga0ANTU1dqbXk8Q1HAjy7L4WHtx+nNoTXWSmJvOhDRV8YuNCVs7LjXV5IjET0UNMa+0PgR8CGGP+BjgdjaJEANr6Bnl01yke2XWCtr4hFhZl8eX3r+QjGxaQl5Ua6/JEYi7SUSgl1to2Y0wlof7vjdEpSxLZGye7+PH24/xy7xmGA5ZNy9z83Yeq2LTMransImNEOg78sXAf+DDwR9ba7ijUJAnIWsvOxk7++YWjbG/wkJOewsc3LuQTGxey2K3hfyLjibQL5fpoFSKJyVrLb+s7+NbzR3n9eBfunHS+/P6V3HllJa50zTMTmYx+QyQmrLW8eLiNbz1fT92pbspyM7j/lsu488pKMlKTY12eiCMowGVWBYOW5w608u0Xj7KvqZfy/Ey+9sHVfHhDBekpCm6R6VCAy6yw1rLtQCv/uO0Ih1r6WFiUxdc/dDkfvKJcq/+JzJACXC65g2d6+ep/HuDVeg+Li7P5//9jLbdcPl87sotESAEul0yHd4h/3HaEn752ktzMVP761lX83lWVanGLRIkCXKJuaCTAg9uP88/P1zMwHOCua6r4/E1Lyc/SLu0i0aQAl6gZ7ef+2i8PcsLTz40rSvirm1dqGVeRS0QBLlFx8EwvX3nmANsbPCwtcfHgp69k0zJ3rMsSiWsKcInI0EiAbz1/lO+93EhuRgpfuW0VH72yUg8oRWaBAlxmbF9TD1/4+ZscaunjIxsq+PL7L9MiUyKzSAEu0zYcCPKdF+v59gv1FGan8cO7arhpZWmsyxJJOApwmZbDLX38+c/r2NfUy+3r5nP/ras0ukQkRhIuwBvavfQPBVhTkRfrUhxlJBBk628a+ea2o+RkpPC9j2/gvavLYl2WSEJLuAD/0uN76egb4oUvbI51KY5R3+blCz9/k7pT3dy8poyv3LZa25eJzAEJFeAD/gBvnOwiELQMDge06t1FWGt5eNdJvvrMATLTkvnWR9dzy+XztMu7yByRUAG+52QXw4HQtpzHOnzaT3ESQyMB/s+T+/n32lNsXu7m6x++nJKcjFiXJSJjJFSA72jwnH1d3+ZVgE+grXeQP3x4N3tOdvO5d1bzZ+9apq3MROagxArwRg+XzcvlUEsv9W3eWJczJ9Wd6uazP6mld2CE737sCm5eMy/WJYnIBBJmuly/f4Q3T3WzabmbysIsBfg4Htt9mjv+ZQepyUk8fu81Cm+ROS5hWuC1x7sYCVo2Li7iaGufAnyMkUCQv/nlIX706jGuWVLEt3/vCgqzNbZbZK6LqAVujPmfxpj9xph9xphHjTFz9inXjkYPKUmGmoUFVJfk0NjhZSQQjHVZMdfl83PXv77Gj149xqeureKhT1+p8BZxiBkHuDGmHPgToMZauxpIBu6MVmHRtrPRw9oF+WSnp1Bd4mI4YDnZ2R/rsmLqUEsvt37nt7x+rItvfPhy7rtllRahEnGQSH9bU4BMY0wKkAU0R15S9HmHRnjrdA8bFxcCnF2fOpG7UXY0ePjd727HPxLk3z+7kY/ULIh1SSIyTTMOcGttE/D3wEngDNBjrX3u/POMMVuMMbXGmNr29vaZVxqB1493Egharl5cDLwd4EcTNMC313fwqR+/Rnl+Jk9/7jrWVxbEuiQRmYFIulAKgNuARcB8INsY8/Hzz7PWbrXW1lhra9zu2Czwv7PRQ2qyYcPCUFC50lOYl5dBQwIG+Kv1HXz6wdepLMzi0S0bKc2ds48tROQiIulC+R3gmLW23Vo7DDwOXBOdsqJrZ4OHdQvyyUx7e+p8dYkr4Vrgrxxp59M/fp2qomwevXsjxVrPRMTRIgnwk8BGY0yWCS2OcRNwMDplRU/v4DB7m3q4enHROcerS1w0tHsJBm2MKptdLx9p5w8eqmVRcTb/dvdGLUYlEgci6QPfBfwC2APsDX/W1ijVFTW1xzsJWti45MIA7/cHONM7GKPKZs+Lh9u4+6Faqt0uHr17o4YJisSJiCbyWGvvA+6LUi2XxI4GD2nJSVxx3oO6pSU5ABxt7aM8PzMWpc2K5w+2cs/De1hW5uLhz1ylzRdE4kjcD/rd0ehhfWX+BUvHJsJQwv860MofPryb5WU5PPKZjQpvkTgT1wHeMzDM/uZerj6v+wSgMDuNwuy0uA3w5/a3cM8ju7lsXi4P/8FV2mxYJA7F9Voorx3rxFrYuPjCAIdQKzweA3zbgVbufWQPq8vzeOgzV5KbofAWiUdx3QLf0eAhPSWJ9ZX54/58dCihtfEzEuVQSy+f/+kbrJqfy08U3iJxLa4DfGejhw0LC0hPGX/rtKUlLnoGhunw+me5skujy+fn7odqyclI4fufrCFH4S0S1+I2wLv7/Rxs6Z2w+wTi60HmSCDI5x7dQ2vPEN/7+AZKNMNSJO7FbYDvbAz1f4/3AHPU2wHeN1tlXTL/71eHeLXew1c/uFprm4gkiDgOcA8ZqUmsrRi//xugLDcDV3qK41vgj+0+zQ9/e4zfv6aKO7SqoEjCiOsAr1lYSFrKxP+JxhiWlLiob3dugL95qpsvPbGXqxcX8b/evzLW5YjILIrLAPd4hzjU0jdp98moareLo63ODPC2vkE++5PduF3pfOdjV5CqzRhEEkpc/sbvOtYJTDz+e6ylpS7a+oboGRi+1GVFlX8kyL0P76F7wM/WT27Q+iYiCSguA3xno4estGQur8i76LnVbmeORLnv6f3UnujiGx9ey6r5F//vFJH4E5cBvqPBQ01V4ZS6FJaWhgLcSZs7PLzzBI++dpJ7Ni/hlrXzY12OiMRI3AV4e98QR9u8F6z/PZGKgizSUpIc8yDztWOd3P/0fjYvd/OFdy+PdTkiEkNxF+C7jnmAycd/j5WcZFhcnM3R1rk/FrzDO8S9j+ymsjCLf7pzPclJJtYliUgMxV2A72jw4EpPYfX83Cm/Z2lpzoxa4Mc6fNSd6p72+2bqK88coGdgmO99YgN5mZomL5Lo4i/AGz28o6qAlGkMqat2uzjdNcCAPzCta/35z+r4zI9fn5Vt2V4+0s5Tdc3cs7maZaU5l/x6IjL3xVWAt/UO0tjum3L3yajqEhfWQsM0WuEnPf3sOdmNx+dnf3PvdEudlgF/gC8/uZfFxdncu3nJJb2WiDhHXAX4jsZQ//dUxn+PdXYkyjQC/Km6prOvXz7SNq3rTdc/PX+UU50DfO2Day7YWUhEEteMA9wYs9wYUzfmT68x5k+jWdx07Wz0kJORMu1x0VVF2SQnmSnPyLTW8mRdE1cuKmR1eS4vH2mfSblTcvBML9//TSMf2VAx7X9ZiEh8i2RX+sPW2nXW2nXABqAfeCJqlc3AzsZOrlpUOO3RGWkpSSwsypryZJ79zb00tPu4fV05m5a52XOy+5LM5AwELV96fC95man81c1a50REzhWtLpSbgAZr7Ykofd60tfYOcqzDN+3uk1HVbhdHp7is7FN1TaQmG25eU8amZSUEgpbt9R0zuu5kHtl1grpT3fzvD6ykQFPlReQ80QrwO4FHx/uBMWaLMabWGFPb3n7puhqOdfgAWFE29eGDY1WXuDjh6Wc4EJz0vEDQ8vSbzWxeXkJ+VhrrK/PJSU+JejdKS88gX3/2MNcvLeb2deVR/WwRiQ8RB7gxJg24Ffj5eD+31m611tZYa2vcbnekl5tQpy+0LVpxzsxaqktLXYwELSc8vknP29XoobV3iNvWhaawpyYncW11Ma8caY/q3pr3P72f4UCQr96+GmM0YUdELhSNFvj7gD3W2tYofNaMebxDADNela/aHRpbfbEHmU/WNeFKT+F3VpaePbZpuZvmnsGoLYi17UArz+5v4U9uWsrCouyofKaIxJ9oBPhHmaD7ZDZ5wi3wwqyZBfiSklBQThbCg8MBfrWvhfesKjtnON8Ny0L/sohGN4pvaIT7ntrH8tIcttywOOLPE5H4FVGAG2OygXcBj0ennJnzeP3kZ6VOawbmWFlpKZTnZ3J0kgB/6XAbfYMjZ7tPRpXnZ1Jd4opKgP/Dc0c40zvI3/zuGm3QICKTiighrLU+a22RtbYnWgXNVKfPT1GEIzWqS1yTtsCffKOZYlc614wzHnvTMje7Gjvp94/M+Pp7T/fw4+3H+NhVlWxYqI2JRWRycdPE6/AOUeRKj+gzlpa4aGj3EhhnbZOegWFeONTGLWvnjdvK37TMjT8QZFdj54yuPRII8sXH36LYlc5fvHfFjD5DRBJL3AS4J0ot8KGRIE1dAxf87Nl9Z/AHghMO6btyUSEZqUkz7kZ5cMcJ9jf3cv+tq8jN0EqDInJxcRPgnT4/Ra7IAxygvv3CCT1P1TVTVZQ14TZtGanJbFxcxCszCPDB4QAPvFTPddXFvG912bTfLyKJKS4CPBC0dPX7KcyOrAtlNMDPH0rY0jPIjkYPt60rn3RM9qZlbho7fJz09E/ruj+vPUWH18/nbqzWmG8RmbK4CPCufj/WQnGELfD8rDSKXekXPMj8jzebsRZuXz/5jMhNo8MJj069FT4SCPIvrzRyRWU+Vy0qnH7RIpKw4iLAPd7wGPAorBdSXZJ9wVDCJ+uaWFuRx6LiySfVLCrOZkFhJi8fnnqA/8dbzZzuGuDezWp9i8j0xEeA+0KzMIsi7EIBWFqSQ0Ob9+y0+Pq2PvY393LrFNYjMcZww1I3Oxo68I9MvqYKQDBoeeClBpaVurhxRUnEtYtIYomPAA+3wCN9iAmhfvC+oRHa+kJ/KTxV10ySgVvWzpvS+zctc+PzB9h9ouui575wqI0jrV7u2byEJG1QLCLTFBcBPrqQVaTDCCE0FhxCDzKttTxV18y11cWU5GRM6f3XVBeTkmQuOpzQWst3X6qnoiCTWy6fP+m5IiLjiYsA93iHSDKhh5CROjuUsK2PPSe7OdnZz23TWM7VlZ5CTVXBRQN817FO9pzsZssNi2c8/V9EEltcJEeHz09BVtq0d+IZjzsnnZyMFI62eXm6ron0lCTes6r04m8cY9OyEg6e6aW1d3DCcx54qYFiVxp31CyItGQRSVBxEeCd3sgn8YwyxrC0xMXhlj6eeesMv7OylJxpzowcHU440aSefU09vHyknU9du0ibFIvIjMVFgHt8Q1EZQjiqusRF7YkuPD7/BSsPTsXKeTm4c9J55ej426w98HIDOekpfOLqhZGWKiIJLE4C3B/xQlZjjfaD52Wmsnn59If3GWPYtMzNb462X7Aw1rEOH7/ae4aPbVyoNU9EJCLxEeDeyBeyGmtpSWh3npvXlJGWMrNbdMMyN939w7x1uvuc41tfaSAlOYlPX1cVaZkikuAcH+DDgSA9A8NRmcQzan1lPldU5vPJq6tm/BnXVxdjzLm79LT2DvLY7ibuqKmY8rBEEZGJOD7Au3zRm8QzKj8rjcfvvZaV82a2wz1AQXYaayvyzwnwH/ymkZFgkC3XL4lGmSKS4Bwf4J4oTuKJtk3L3Lx5qpvufj/d/X4e2XWSW9bOp7IoK9aliUgccH6An51GH70ulGjZtNxN0MJv6zt4aMcJ+v0B7tms1reIREdKJG82xuQDPwBWAxb4tLV2RzQKm6rRhayiOYwwWtZW5JOXmcqv9rWwvb6Dm1aUsKJs5t0yIiJjRRTgwD8Bz1prP2yMSQNmvW9gtAUe6Vrgl0JykuH6pcU889YZALW+RSSqZtyFYozJA24AfghgrfVba7snf1f0eXxDJCeZOTumenRW5pVVhdRUacMGEYmeSPrAFwHtwL8aY94wxvzAGHPBjgfGmC3GmFpjTG17+8w2/J1Mp89PYXbanF2O9cYVJVSXuPjzdy+LdSkiEmciCfAU4ArgAWvtesAHfPH8k6y1W621NdbaGrfbHcHlxtcR5Uk80VbkSue//mwTVy0uinUpIhJnIgnw08Bpa+2u8Pe/IBTosyoau9GLiDjRjAPcWtsCnDLGLA8fugk4EJWqpsHjHYrqLEwREaeIdBTKHwOPhEegNAKfiryk6fGE+8BFRBJNRAFura0DaqJUy7QNjQToGxyZk0MIRUQuNUfPxBzdC7NQXSgikoAcHeDR3I1eRMRpnB3gc3ghKxGRS83RAd4ZXgdlLi5kJSJyqTk6wEe7UDQKRUQSkbMD3OcnNdmQmxHpaEgREedxdoCHJ/EYMzfXQRERuZQcHeCdmsQjIgnM0QHe4dU6KCKSuBwd4B7fkIYQikjCcnSAd3r9GkIoIgnLsQE+OBzA5w+oD1xEEpZjA3x0FqYWshKRROXcAPeO7kavLhQRSUzODXCfFrISkcTm3AAPT6MvVgtcRBKUYwN8dCGrQrXARSRBOTbAPV4/6SlJZKclx7oUEZGYcGyAd3j9FGWnaR0UEUlYES3jZ4w5DvQBAWDEWjtr+2N2+oY0iUdEElo01mF9p7W2IwqfMy3ajV5EEp1ju1A8WshKRBJcpAFugeeMMbuNMVvGO8EYs8UYU2uMqW1vb4/wcm/z+IYoVheKiCSwSAP8OmvtFcD7gD8yxtxw/gnW2q3W2hprbY3b7Y7wciH9/hEGh4PqQhGRhBZRgFtrm8Jf24AngCujUdTFjE7i0VKyIpLIZhzgxphsY0zO6Gvg3cC+aBU2mQ7v6G70CnARSVyRjEIpBZ4Ij8NOAf7NWvtsVKq6iM7RdVA0jV5EEtiMA9xa2wisjWItUzbahaI+cBFJZI4cRqiVCEVEnBrg3iEyU5PJSovGPCQREWdyZIB3+jSJR0TEkQHe4dNmxiIijgzwTt+QxoCLSMJzZIB7wkvJiogkMscFuLUWj9evnXhEJOE5LsC9QyP4A0HthSkiCc9xAa5JPCIiIc4LcE3iEREBnBjgowtZqQtFRBKc4wK8Uy1wERHAgQE+2oWiPnARSXTOC3CvH1d6ChmpybEuRUQkppwX4L4hdZ+IiODEAPf61X0iIoITA9zn1wgUERGcGOBeLWQlIgIOC3BrrdYCFxEJc1SA9w6MMBK0WgtcRIQoBLgxJtkY84Yx5ploFDQZj290FqZa4CIi0WiBfx44GIXPuSitgyIi8raIAtwYUwG8H/hBdMqZnFYiFBF5W6Qt8G8CfwEEJzrBGLPFGFNrjKltb2+P6GKjXSjF6gMXEZl5gBtjPgC0WWt3T3aetXartbbGWlvjdrtnejng7RZ4QZZa4CIikbTArwVuNcYcB34K3GiMeTgqVU2g0+cnNyOFtBRHDZ4REbkkZpyE1tovWWsrrLVVwJ3AC9baj0etsnF0eIc0hFBEJMxRTdlOn3ajFxEZFZUAt9a+ZK39QDQ+azIer2ZhioiMclQL3OPzU6iFrEREAAcFeDBo6er3U6wWuIgI4KAA7xkYJhC0msQjIhLmmAA/uw6KRqGIiAAOCvCO8CQejUIREQlxTIB3aiErEZFzOCbAPd5QF4r6wEVEQpwT4OEWeKHWQRERAZwU4F4/BVmppCQ7pmQRkUvKMWnY6fOr+0REZAzHBLgWshIROZdjAlwLWYmInMsxAe7xaSErEZGxHBHggfA6KFrISkTkbY4I8K5+P9aihaxERMZwRIBrN3oRkQs5I8BHF7JSF4qIyFnOCPBwC1xdKCIib3NEgI8uZKUuFBGRt804wI0xGcaY14wxbxpj9htj/jqahY3l8Q6RZCBf66CIiJyVEsF7h4AbrbVeY0wq8FtjzK+stTujVNtZHp+fgqw0kpNMtD9aRMSxZhzg1loLeMPfpob/2GgUdT7tRi8icqGI+sCNMcnGmDqgDdhmrd01zjlbjDG1xpja9vb2GV1nTUUeN60sjaRUEZG4Y0IN6Qg/xJh84Angj621+yY6r6amxtbW1kZ8PRGRRGKM2W2trTn/eFRGoVhru4EXgfdG4/NEROTiIhmF4g63vDHGZALvAg5FqzAREZlcJKNQ5gEPGmOSCf1F8DNr7TPRKUtERC4mklEobwHro1iLiIhMgyNmYoqIyIUU4CIiDqUAFxFxKAW4iIhDRWUiz5QvZkw7cGKGby8GOqJYzmxzcv1Orh2cXb+TawL2hDwAAAO3SURBVAdn1z+Xal9orXWff3BWAzwSxpja8WYiOYWT63dy7eDs+p1cOzi7fifUri4UERGHUoCLiDiUkwJ8a6wLiJCT63dy7eDs+p1cOzi7/jlfu2P6wEVE5FxOaoGLiMgYCnAREYdyRIAbY95rjDlsjKk3xnwx1vVMhzHmuDFmrzGmzhgz53ezMMb8yBjTZozZN+ZYoTFmmzHmaPhrQSxrnMgEtd9vjGkK3/86Y8zNsaxxMsaYBcaYF40xB8IbhX8+fHzO3/9JanfE/Z9ok3ZjzCJjzK5w9vy7MWZO7e045/vAw8vVHiG03vhp4HXgo9baAzEtbIqMMceBGmvtXJkQMCljzA2E9jp9yFq7Onzs60CntfZvw3+BFlhr/zKWdY5ngtrvB7zW2r+PZW1TYYyZB8yz1u4xxuQAu4Hbgd9njt//SWq/Awfcf2OMAbLHbtIOfB74M+Bxa+1PjTHfA9601j4Qy1rHckIL/Eqg3lrbaK31Az8FbotxTXHLWvsK0Hne4duAB8OvHyT0iznnTFC7Y1hrz1hr94Rf9wEHgXIccP8nqd0RbMh4m7TfCPwifHzO3XsnBHg5cGrM96dx0P8YhP4neM4Ys9sYsyXWxcxQqbX2TPh1C+C0HaY/Z4x5K9zFMue6H8ZjjKkitN7+Lhx2/8+rHRxy/8/fpB1oALqttSPhU+Zc9jghwJ3uOmvtFcD7gD8K/zPfsWyoz21u97ud6wFgCbAOOAP8Q2zLuThjjAt4DPhTa23v2J/N9fs/Tu2Ouf/W2oC1dh1QQehf/itiXNJFOSHAm4AFY76vCB9zBGttU/hrG/AEof8xnKY13Mc52tfZFuN6psxa2xr+xQwC32eO3/9w/+tjwCPW2sfDhx1x/8er3Wn3H87ZpP1qIN8YM7pz2ZzLHicE+OvA0vDT4DTgTuDpGNc0JcaY7PADHYwx2cC7gX2Tv2tOehq4K/z6LuCpGNYyLaPBF/ZB5vD9Dz9I+yFw0Fr7j2N+NOfv/0S1O+X+T7BJ+0FCQf7h8Glz7t7P+VEoAOGhR98EkoEfWWu/FuOSpsQYs5hQqxtC+4/+21yv3RjzKLCZ0FKarcB9wJPAz4BKQssB32GtnXMPCyeofTOhf75b4Djw2TH9yXOKMeY64DfAXiAYPvxXhPqS5/T9n6T2j+KA+2+MuZzQQ8qxm7T/3/Dv8E+BQuAN4OPW2qHYVXouRwS4iIhcyAldKCIiMg4FuIiIQynARUQcSgEuIuJQCnAREYdSgIuIOJQCXETEof4b7iTRVSnt/BYAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "plt.plot(scores)\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "n3KgR-19nJ9Q"
      },
      "outputs": [],
      "source": [
        "agent.env.close()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "EPR0kXmMnJ9R"
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "colab": {
      "name": "DQN_Atari_v2.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "accelerator": "GPU"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}